package collector;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import seotools.positions.Positions;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URL;
import java.util.*;
import java.util.regex.Pattern;

public class HostingTest {
//    public static String yandexCookie = "yandexuid=2027961314420485; fuid01=4d726ae12b1a1ac0.N9hJY3OOtWG-kEtar5NvfvMUs1Xo_8AUqbE-fG1uUYhwf2PZq1e_rMNyTIPcAliV5rOj5oRQaNMCW7Z2qn2HXaeA_Otjy40U2yeL3hFKkrbBAo6y88nb2eM2jOvZk-KQ; yabs-frequency=/3/z7a06ciAF0000NS201TC2jpBw04fC0gLhxO1A1CAN0000W00/; yp=1630729273.sp.nd:50:lang:!#1635007019.udn.c2hlbXBsaW5lcg==; my=YycCAAAtAQI2AQEA; L=dVIMOXYCcgVAc3pjfVBSR3FjdnpAcwABXT0qUgEwbDApBSFcAjdFDVBfPAQtGCsKWgEFEwdIDiBYMFF1NAZ8IQ==.1319647019.9153.293247.27456d6039173e801b32231707405957; aw=1_keJxiFGVg+nZVkCNFu5+l8oKiwAGN9Sz/AQAAAP//AwBB6Acz#A#; yandex_login=shempliner; Session_id=1319647019.-40297.3.118550753.2:330838428:230.8:1319647019192:1597472257:13.79373.11703.d5496652cb2ced4a06b96a480c5a47e8; t=p";

    public static void main(String[] args) throws IOException, InterruptedException {
//        Map<String, Integer> rank = new HashMap<String, Integer>();
        SortableValueMap<String, Integer> rank = new SortableValueMap<String, Integer>();
        long hostCount = 0;

        int[] regions = new int[]{2, 213, 65, 54, 47};

//        List<String> phrases = new ArrayList<String>();
//        phrases.add("танцевальная студия");
//        phrases.add("доставка пиццы");
        List<String> phrases = Positions.loadPhrases("checkhosting2.txt");

        phrases = Positions.processPhrases(phrases);
//        int region = 2; //spb
//        int region = 213; //msk

        for (int region : regions) {

            for (String phrase : phrases) {
                Map<Integer, String> positions = Positions.requestYandexPositions(phrase, region, 8);
                for (String url : positions.values()) {
                    String host = new URL(url).getHost();
                    Document doc = null;
                    while (true) {
                        try {
                            doc = Jsoup.connect("https://www.nic.ru/whois/?query=" + host).timeout(20000).get();
                            break;
                        } catch (IOException e) {
                            Thread.sleep(5000);
                        }
                    }
                    String whois = doc.select("div#whois").text();

                    String nserver = "nserver:";
                    int idx1 = whois.indexOf(nserver);
                    int idx2 = whois.indexOf(nserver, idx1 + 1);
                    if (idx1 == -1) {
                        nserver = "Name Server:";
                        idx1 = whois.indexOf(nserver);
                        idx2 = whois.indexOf(nserver, idx1 + 1);
                    }
                    String hosting = null;
                    try {
                        hosting = whois.substring(idx1 + nserver.length(), idx2).replace((char) 160, (char) 32).trim().toLowerCase();
                    } catch (Exception e) {
                        System.err.println("Unknown hosting https://www.nic.ru/whois/?query=" + host);
                        continue;
                    }
                    if (hosting.endsWith(".")) {
                        hosting = hosting.substring(0, hosting.length() - 1);
                    }
                    if (rank.containsKey(hosting)) {
                        int newRank = rank.get(hosting) + 1;
                        rank.put(hosting, newRank);
                        hostCount++;
                    } else {
                        rank.put(hosting, 1);
                        hostCount++;
                    }
                    Thread.sleep(3000);
                }
            }
            rank.sortByValue();
            PrintStream out = new PrintStream(new FileOutputStream("hosting/region" + region + ".txt"));
            int count = 0;
            for (String hosting : rank.keySet()) {
                count++;
//            System.out.println(hosting + ": \t" + rank.get(hosting) + "; \t" + (int)((rank.get(hosting) * 100) / hostCount) + "%");
                out.println(count + ". " + hosting + ": \t" + rank.get(hosting) + "; \t" + (int) ((rank.get(hosting) * 100) / hostCount) + "%");
            }
            out.close();
        }
    }

    static class SortableValueMap<K, V extends Comparable<V>>
            extends LinkedHashMap<K, V> {
        public SortableValueMap() {
        }

        public void sortByValue() {
            List<Map.Entry<K, V>> list = new LinkedList<Map.Entry<K, V>>(entrySet());

            Collections.sort(list, new Comparator<Map.Entry<K, V>>() {
                public int compare(Map.Entry<K, V> entry1, Map.Entry<K, V> entry2) {
//                    return entry1.getValue().compareTo(entry2.getValue());
                    return entry2.getValue().compareTo(entry1.getValue());
                }
            });

            clear();

            for (Map.Entry<K, V> entry : list) {
                put(entry.getKey(), entry.getValue());
            }
        }
    }
}
